package com.kd.crawler.parser.thread;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Properties;

import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSON;
import com.kd.crawler.common.jms.ActiveMQProducer;
import com.kd.crawler.common.utils.MD5Util;
import com.kd.crawler.common.utils.PropertiesUtils;
import com.kd.crawler.dao.OriginalHtmlDao;
import com.kd.crawler.dao.StatisticEntryUrlDao;
import com.kd.crawler.entity.CrawlerEntry;
import com.kd.crawler.entity.Document;
import com.kd.crawler.parser.HtmlParser;
import com.kd.crawler.parser.HtmlParserImpl;
import com.kd.crawler.parser.StandardAnalyzer;

public class PageParserThread implements Runnable {
	
	private static Logger logger=Logger.getLogger(PageParserThread.class);
	Properties properties = PropertiesUtils.getProperties("activeMQ.properties");
	
//	private HtmlParser parser = new HtmlParserImpl();
	private HtmlParser parser;
	private HtmlParser standardParser = new StandardAnalyzer();
	private CrawlerEntry cr = null;
	private PageParserListener<PageParserThread> listener = null;
	
	private static HashMap<String,HtmlParser> parserCache=new HashMap<String,HtmlParser>();
	
	public PageParserThread(PageParserListener<PageParserThread> listener) {
		this.listener = listener;
	}

	public void run() {
		try{
			doParse();
		}finally{
			try{
				cr.setHtml("");
				logger.info("发送到已完成队列中:"+JSON.toJSONString(cr));
//				MQMessageSender.offerQueueMessage(properties.getProperty("QUEUE_CRAWL_FINISH_ENTRY_MESSAGE_OLD").trim(), JSON.toJSONString(cr));				
			}catch(Exception e){
				e.printStackTrace();
			}
			listener.onFinish(this);
		}
	}
	
	private void doParse(){
		if(null == cr) {
			return ;
		}	
		if(cr.getStatusCode()!=200){
			cr.setHtml(null);
			logger.error("访问地址报错("+cr.getStatusCode()+"):"+JSON.toJSONString(cr));
			return;
		}
		
		/*
		 * this is old
		 * */
		HtmlParser parser=parserCache.get(cr.getParserName());
		if(parser==null){
			parser= new HtmlParserImpl(cr.getParserName());
			parserCache.put(cr.getParserName(),parser);
		}
		
		
		try {
			if(!cr.isDetailed()) {
				String url=cr.getUrl();
				int statusCode=cr.getStatusCode();
				logger.info("start");
				//old version
				List<String> links = parser.getDetailedInnerLinks(cr);
				//new version
//				List<String> links = standardParser.getDetailedInnerLinks(cr);
				
				if(null==links || links.size()<10){
					String html=cr.getHtml();
					cr.setHtml(null);
					logger.error("解释详细页地址不正常("+cr.getStatusCode()+"):"+JSON.toJSONString(cr));
					cr.setHtml(html);
				}
				logger.info("本次详细页地址数量为:"+links.size());
				ArrayList<CrawlerEntry> crawlerEntries=new ArrayList<CrawlerEntry>();	
				for(String link : links) {
					CrawlerEntry crawlerEntry=new CrawlerEntry();
					crawlerEntry.setCrawlTime(cr.getCrawlTime());
					crawlerEntry.setDetailed(true);
					crawlerEntry.setEntryId(cr.getEntryId());
					crawlerEntry.setIntervalTime(cr.getIntervalTime());
					crawlerEntry.setJobID(cr.getJobID());
					crawlerEntry.setLevel(cr.getLevel());
					crawlerEntry.setParserName(cr.getParserName());
					crawlerEntry.setRegionId(cr.getRegionId());
					crawlerEntry.setStatusCode(0);
					crawlerEntry.setTypeId(cr.getTypeId());
					crawlerEntry.setUrl(link);					
					crawlerEntries.add(crawlerEntry);					
				}
				if(crawlerEntries.size()>0){
//					StatisticEntryUrlDao.getInstance().insert(cr, crawlerEntries.size(), "N");
					URLFilterQueue.offer(crawlerEntries);
				}
				cr.setDetailed(false);
				cr.setUrl(url);
				cr.setStatusCode(statusCode);
				//reuse for new version
//				MQMessageSender.offerQueueMessage(properties.getProperty("QUEUE_CRAWL_FINISH_ENTRY_MESSAGE_NEW").trim(), JSON.toJSONString(cr));			
			} else {
				//here, want to add database
//				OriginalHtmlDao.getInstance().insert(MD5Util.MD5(cr.getUrl()), cr.getUrl(),cr.getHtml());
				//
				//old version
				Document meta = parser.parse(cr);
				//new version
//				Document meta = standardParser.parse(cr);
//				StatisticEntryUrlDao.getInstance().update(cr, meta, "N");
//				meta.setAddressValue(cr.getRegionId());
//				meta.setEntryID(cr.getEntryId());
//				meta.setJobId(cr.getJobID());
//				meta.setSource(cr.getJobID());
//				meta.setUrl(cr.getUrl());
//				meta.setType(cr.getTypeId());
//				meta.setLevel(cr.getLevel());
//				logger.debug("解析出的文档"+cr.getUrl());
				logger.debug(cr.getUrl() + ", 解释出的文档："+JSON.toJSONString(meta));
//				if(meta!=null&&!StringUtils.isBlank(meta.getContent())&&!StringUtils.isBlank(meta.getTitle())&&meta.getDate()!=null&&meta.getDate()>0){
//					String summary = meta.getContent().replaceAll("<script[^<>]*>[^<>]*</script[^<>]*>", "").replaceAll("<style[^<>]*>[^<>]*</style[^<>]*>", "").replaceAll("<[^<>]*>", "").replaceAll("[\\s　]", "");
//					meta.setSummary(summary.length() > 200 ? summary.substring(0, 200) : "");
//					long docId=DocumentDao.getInstance().insert(meta);
//					DocumentContentDao.getInstance().insert(docId, meta.getContent());
//				}
				//ActiveMQProducer.sendTopicMessage(properties.getProperty("QUEUE_CRAWL_DOC_MESSAGE").trim(), JSON.toJSONString(meta));
//				MQMessageSender.offerTopicMessage(properties.getProperty("QUEUE_CRAWL_DOC_MESSAGE").trim(), JSON.toJSONString(meta));		
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	
	public CrawlerEntry getCrawlerEntry() {
		return cr;
	}

	public void setCrawlerEntry(CrawlerEntry cr) {
		this.cr = cr;
	}

	public static void main(String args[]){
		Properties properties = PropertiesUtils.getProperties("activeMQ.properties");
		ActiveMQProducer.sendTopicMessage(properties.getProperty("QUEUE_CRAWL_DOC_MESSAGE").trim(), "ok");
		System.out.println("OK");
	}

}


